<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Segment Merging | Elasticsearch: The Definitive Guide [2.x] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: The Definitive Guide [2.x]">
<link rel="up" href="inside-a-shard.html" title="Inside a Shard">
<link rel="prev" href="translog.html" title="Making Changes Persistent">
<link rel="next" href="search-in-depth.html" title="Search in Depth">
<meta name="DC.type" content="Learn/Docs/Legacy/Elasticsearch/Definitive Guide/2.x">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<p>
  <strong>WARNING</strong>: The 2.x versions of Elasticsearch have passed their
  <a href="https://www.elastic.co/support/eol">EOL dates</a>. If you are running
  a 2.x version, we strongly advise you to upgrade.
</p>
<p>
  This documentation is no longer maintained and may be removed. For the latest
  information, see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html">current
  Elasticsearch documentation</a>.
</p>
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: The Definitive Guide [2.x]</a></span>
»
<span class="breadcrumb-link"><a href="getting-started.html">Getting Started</a></span>
»
<span class="breadcrumb-link"><a href="inside-a-shard.html">Inside a Shard</a></span>
»
<span class="breadcrumb-node">Segment Merging</span>
</div>
<div class="navheader">
<span class="prev">
<a href="translog.html">« Making Changes Persistent</a>
</span>
<span class="next">
<a href="search-in-depth.html">Search in Depth »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="merge-process"></a>Segment Merging<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/075_Inside_a_shard/60_Segment_merging.asciidoc">edit</a>
</h2>
</div></div></div>
<p>With the automatic refresh process creating a new segment every second, it
doesn’t take long for the number of segments to explode. Having too many
segments is a problem. Each segment consumes file handles, memory, and CPU
cycles.  More important, every search request has to check every segment in
turn; the more segments there are, the slower the search will be.</p>
<p>Elasticsearch solves this problem by merging segments in the background. Small
segments are merged into bigger segments, which, in turn, are merged into even
bigger segments.</p>
<p>This is the moment when those old deleted documents are purged from the filesystem.  Deleted documents (or old versions of updated documents) are not
copied over to the new bigger segment.</p>
<p>There is nothing you need to do to enable merging. It happens automatically
while you are indexing and searching. The process works like as depicted in <a class="xref" href="merge-process.html#img-merge" title="Two commited segments and one uncommited segment in the process of being merged into a bigger segment">Figure 25, “Two commited segments and one uncommited segment in the process of being merged into a bigger segment”</a>:</p>
<div class="olist orderedlist">
<ol class="orderedlist">
<li class="listitem">
While indexing, the refresh process creates new segments and opens them for
search.
</li>
<li class="listitem">
<p>The merge process selects a few segments of similar size and merges them
into a new bigger segment in the background. This does not interrupt
indexing and searching.</p>
<div id="img-merge" class="imageblock">
<div class="content">
<img src="images/elas_1110.png" alt="Two commited segments and one uncommited segment in the process of being merged into a bigger segment">
</div>
<div class="title">Figure 25. Two commited segments and one uncommited segment in the process of being merged into a bigger segment</div>
</div>
</li>
<li class="listitem">
<p><a class="xref" href="merge-process.html#img-post-merge" title="Once merging has finished, the old segments are deleted">Figure 26, “Once merging has finished, the old segments are deleted”</a> illustrates activity as the merge completes:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
The new segment is flushed to disk.
</li>
<li class="listitem">
A new commit point is written that includes the new segment and
excludes the old, smaller segments.
</li>
<li class="listitem">
The new segment is opened for search.
</li>
<li class="listitem">
The old segments are deleted.
</li>
</ul>
</div>
<div id="img-post-merge" class="imageblock">
<div class="content">
<img src="images/elas_1111.png" alt="Once merging has finished, the old segments are deleted">
</div>
<div class="title">Figure 26. Once merging has finished, the old segments are deleted</div>
</div>
</li>
</ol>
</div>
<p>The merging of big segments can use a lot of I/O and CPU, which can hurt
search performance if left unchecked.  By default, Elasticsearch throttles the
merge process so that search still has enough resources available to perform
well.</p>
<div class="tip admon">
<div class="icon"></div>
<div class="admon_content">
<p>See <a class="xref" href="indexing-performance.html#segments-and-merging" title="Segments and Merging">Segments and Merging</a> for advice about tuning merging for your use
case.</p>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="optimize-api"></a>optimize API<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/075_Inside_a_shard/60_Segment_merging.asciidoc">edit</a>
</h3>
</div></div></div>
<p>The <code class="literal">optimize</code> API is best described as the <em>forced merge</em> API. It forces a
shard to be merged down to the number of segments specified in the
<code class="literal">max_num_segments</code> parameter. The intention is to reduce the number of
segments (usually to one) in order to speed up search performance.</p>
<div class="warning admon">
<div class="icon"></div>
<div class="admon_content">
<p>The <code class="literal">optimize</code> API should <em>not</em> be used on a dynamic index—​an
index that is being actively updated.  The background merge process does a
very good job, and optimizing will hinder the process. Don’t interfere!</p>
</div>
</div>
<p>In certain specific circumstances, the <code class="literal">optimize</code> API can be beneficial.
The typical use case is for logging, where logs are stored in an index per
day, week, or month.  Older indices are essentially read-only; they are
unlikely to change.</p>
<p>In this case, it can be useful to optimize the shards of an old index down to
a single segment each; it will use fewer resources and searches will be
quicker:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">POST /logstash-2014-10/_optimize?max_num_segments=1 <a id="CO41-1"></a><i class="conum" data-value="1"></i></pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO41-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Merges each shard in the index down to a single segment</p>
</td>
</tr>
</table>
</div>
<div class="warning admon">
<div class="icon"></div>
<div class="admon_content">
<p>Be aware that merges triggered by the <code class="literal">optimize</code> API are not
throttled at all. They can consume all of the I/O on your nodes, leaving
nothing for search and potentially making your cluster unresponsive. If you
plan on optimizing an index, you should use shard allocation (see
<a class="xref" href="retiring-data.html#migrate-indices" title="Migrate Old Indices">Migrate Old Indices</a>) to first move the index to a node where it is safe to
run.</p>
</div>
</div>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="translog.html">« Making Changes Persistent</a>
</span>
<span class="next">
<a href="search-in-depth.html">Search in Depth »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
